#!/usr/bin/env python2
# ----------------------------------------------------------------------------
# qrpbbb_scrub - sanitize raw news messages to what's appropriate for QRPBBB
# and being on-air. Removing unnecessary headers added by newsreaders and
# newsservers, and checking the mandated conditions.
# ----------------------------------------------------------------------------
# Re: the mandated conditions: if you think a one-line reply, with 20 lines of
# quoting (and 10 lines of headers) has any content at all, type "cat
# /dev/random" to satisfy your appetite for noise instead.
# ----------------------------------------------------------------------------
# Hey. No-one seriously makes fun of someone who designs and builds a working
# transceiver from scratch using discrete parts. This is the software
# equivalent.
# ----------------------------------------------------------------------------
# Modifying this file will require a newsserver reload.

# TODO before releasing to anyone:
# top-posting check
# handle duplicate headers

import os, time, re, getpass, sys, select
import site_config as cfg
import qrpbbb_lib as lib

# These are the only header records QRPBBB accepts.
# No customs like you could with RFC1036, sorry.
#
ACCEPTHEADERS = ["From", "Date", "Newsgroups", "Subject", "Message-ID",
                 "Path", "Distribution", "Organization", "Followup-To",
                 "Approved", "Expires", "Reply-To", "Keywords", "Summary",
                 "References", "Control", "X-QRPBBB"]

# Very basic, but will do for now..
#
CALLCHECKS = ["VK[0-9][A-Z][A-Z]+", "ZL[0-9][A-Z]+"]

(BADDIST,BADXPOST,BADFROM,BADFILTER,BADQUOTE,BADLENGTH,BADBODY,BADLINE) \
    = (1, 2, 4, 8, 16, 32, 64, 128)
(WARNWHITESPACE, WARNBLANK, WARNREF, WARNLINCONT, WARNHEADERS, WARNNOHEAD) \
    = (256, 512, 1024, 2048, 4096, 8192)

#
# Read the file paths from stdin given by inn. There may be several.
#

# this could possibly use a timeout, I think..
files = []
f = sys.stdin.readline()        # so much for 'several'..
files.append (f.rstrip("\n"))

for f in files:
    m = lib.fetch_messagetext(f).splitlines()

    # no splitlines(keepends=) in Python2.7
    message = []
    for l in m:
        message.append(l + "\n")

    scrubbed = WARNNOHEAD
    headers = {}
    deadheaders = []
    badness = ""
    maxmsglength = cfg.LIMITS["qrpbbb"]["MAXMSGLEN"]
    maxmsgsize = cfg.LIMITS["qrpbbb"]["MAXMSGSIZ"]

    # Header content tests

    li = 0
    longlen = 0
    while li < len(message):
        m = message[li].rstrip("\n")

        if len(m) == 0:         # reached the end of the header lines
            break

        # handle continuation lines for multi-line headers
        if message[li+1][0] == ' ': # "li+1" potential hazard
            scrubbed |= WARNLINCONT
            m = m + message[li+1].rstrip("\n")
            del message[li+1]

        if len(m) > longlen:
            longlen = len(m)
            if len(m) > 99:
                scrubbed |= BADLINE

        (header, payload) = m.split(': ', 1)

        # tests for the various individual header fields
        #
        if header in ACCEPTHEADERS:
            headers[header] = payload
            li += 1

            if header == "X-QRPBBB":
                scrubbed &= (65535-WARNNOHEAD)

            # Mandated condition - specific Distributions
            #
            if header == "Distribution":
                msgdistro = payload
                if payload not in cfg.ACCEPTDISTRO:
                    scrubbed |= BADDIST
                else:
                    maxmsglength = cfg.LIMITS[msgdistro]['MAXMSGLEN']
                    maxmsgsize = cfg.LIMITS[msgdistro]['MAXMSGSIZ']

            # Mandated condition - no crossposting
            # Sites filtering what groups they want isn't mandated from on-high.
            #
            if header == "Newsgroups":
                if "," in payload:
                    scrubbed |= BADXPOST
                for r in cfg.BADGROUPS:
                    if re.match (r, payload, re.IGNORECASE):
                        badness = r + " " + payload
                        scrubbed |= BADFILTER

            # Mandated condition - From must mention a callsign (either as
            # the username, or otherwise included). Filtering "Bad" callsigns
            # from the whole network isn't yet mandated or desirable, but the
            # facility exists.

            # Finding out what the email address is..
            # "(...)" is a comment - remove
            # replace "<" and ">" with whitespace
            # ..then find the largest splitted string with a "@" in it.
            #
            if header == "From":
                scrubbed |= BADFROM
                for r in CALLCHECKS:
                    if re.search (r, payload, re.IGNORECASE):
                        scrubbed &= (65535-BADFROM)
                for r in cfg.BADCALLS:
                    if re.search (r, payload, re.IGNORECASE):
                        badness = r + " " + payload
                        scrubbed |= BADFILTER

                rl = re.split ("<|>|{|}| ", payload)
                email = ""
                for s in rl:
                    if '@' in s and len(s) > len(email):
                        email = s
                if len(email) == 0:
                    scrubbed |= BADFROM

            # Mandated condition: References lists only the preceeding posting.
            # Threading newsreaders will still work. It's to avoid the cascade
            # of needless header lines in a message.
            #
            if header == "References":
                if " " in payload:
                    payload = re.sub ("^.*\<","<",payload)
                    headers[header] = payload
                    scrubbed |= WARNREF
        else:
            scrubbed |= WARNHEADERS
            deadheaders.append (m)
            del message[li]

    # Now for a couple of message body checks..

    bodystart = li+1

    # Mandated: only one blank line between the header and body sections,
    # and no trailing blank lines at the end.
    #
    while len(message) > bodystart and len(message[bodystart]) == 1:
        scrubbed |= WARNBLANK
        del message[bodystart]

    while len(message[-1]) == 1 and len(message) > bodystart:
        scrubbed |= WARNBLANK
        del message[-1]

    # Mandated by RFC1036 (and me) - no message without a body section
    #
    if len(message) == bodystart:
        scrubbed |= BADBODY

    # Mandated: post line limits for the various Distributions.
    #
    if len(message) - bodystart > maxmsglength:
        scrubbed |= BADLENGTH

    # Mandated: no quoting more than 5 lines of text at a time.
    # Mandated: no top-posting (needs a "rm -rf cfg.QRPBBBHOME" system call :P)
    # Mandated: no posts with >4x more quoting than original text.
    # These are all easy to get around manually, but they're really to stop
    # MeToos, Outlook Express users, etc.
    #
    (si, comrun, totquote, seenquotes, botpost) = (0,0,0,0,0)

    for m in message[bodystart:]:
        si += len(m)
        if len(m) > 1:
            n = m.lstrip()
            if len(n) > 0 and n[0] == '>':
                seenquotes = 1
                totquote += 1
                comrun += 1
                if comrun > 5:
                    scrubbed |= BADQUOTE
            else:
                if len(n) > 0 and seenquotes:
                    botpost += 1
                comrun = 0

    if totquote > 4*(len(message[bodystart:]) - totquote):
        scrubbed |= BADQUOTE
    if seenquotes and botpost == 0:
        scrubbed |= BADQUOTE    # actually should have a top-posting error

    # Mandated: body message size limit
    #
    if si > maxmsgsize:
        scrubbed |= BADLENGTH

    # Mandated: refusing to spool irrecoverable violations.
    # It stops Usenet leaking in, and LARTing QRPBBB users who Don't Get It.
    #
    if (scrubbed & (WARNWHITESPACE-1)) and (scrubbed & (BADFROM|BADDIST)) == 0:
        # There's a qrpbbb user to complain to-- prepare an email explaining why
        # their post was dropped.
        #
        m = "Your posting was automatically rejected by the QRPBBB system because...\n\n"
        if scrubbed & BADDIST:
            m += "There is an invalid Distribution header.\n"
        if scrubbed & BADXPOST:
            m += "It is crossposted.\n"
        if scrubbed & BADFROM:
            m += "No recognisable callsign or email address in the From header.\n"
        if scrubbed & BADFILTER:
            m += "It involves a filtered callsign or newsgroup: " + badness + "\n"
        if scrubbed & BADQUOTE:
            m += "There is excessive quoting.\n"
        if scrubbed & BADLENGTH:
            m += "The message is too large/long. "
            m += "(length=%s, maxlength=%s lines, size=%s, maxsize=%s bytes)\n" % \
                (len(message)-bodystart, maxmsglength, si, maxmsgsize)
        if scrubbed & BADBODY:
            m += "There is no text in the message body!\n"
        if scrubbed & BADLINE:
            m += "Longest line length exceeds 99 characters (length=%s).\n" % longlen

        # append to BadList for BADFILTER ..

        m += "\n--- THE OFFENDING MESSAGE FOLLOWS SHOULD YOU NEED TO RESUBMIT IT\n\n"
        m += ''.join(message)

        mail = "/tmp/" + str (os.getpid()) + ".mail"
        foo = open (mail, mode='w')
        foo.write (m)
        foo.close()

        subj = "Your QRPBBB posting was dropped."
        emailstr = "cat {0}|/usr/bin/mail -s '{1}' {2}".format (mail, subj, email)
        print "Dropped message from", email
        os.system (emailstr)
        os.remove (mail)
        os.remove (f)
        continue

    # The message made it through, but was modified, so it needs updating.

    if scrubbed >= WARNWHITESPACE:
        if scrubbed & WARNNOHEAD:
            newmessage = "X-QRPBBB: Scrubbed\n"
        else:
            newmessage = ""
        for h in headers.keys():
            newmessage += h + ": " + headers[h] + '\n'
        newmessage += "\n" + ''.join(message[bodystart:])

        foo = open (f, mode='w')
        foo.write (newmessage)
        foo.close()
        print "Scrubbed", f, scrubbed # this ends up in /var/log/news/errlog

#
# ----------------------------------------------------------------------------
#
